package extractorCode;

import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.FileReader;
import java.io.FileWriter;
import java.io.IOException;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import java.io.*;

public class link4Text {
	
	private final static int INITIAL_BUF_SIZE = 65536;

	public static void main(String args[]) throws Exception
	{
		
		int count=0;
	    BufferedReader reader= new BufferedReader(new FileReader("c:\\keywords-link4-edited.txt"));

	    String keyword;
	    int lastIndex;
	    while((keyword=reader.readLine())!=null)
	    {
	    	
	    	String[] splitarr=keyword.split("\t");
	    	String key = splitarr[0];
	    	int maxPage=Integer.parseInt(splitarr[1]);
	    	System.out.println(keyword);
	       if(maxPage>-1)
	       {
	    	
		
		String urlString;
		BufferedReader br = new BufferedReader(new FileReader("C:\\Users\\Anu\\Desktop\\Website4\\link4-links\\link4-"+key+".txt"));
		count=0;
		while((urlString=br.readLine())!=null)
		{	
		
		System.out.println(urlString);
		String[]splitarr1=urlString.split("\t");
		String url = splitarr1[0];	
		try
		{
		String page = (fetchPage(url));
		System.out.println(page);

		page= page.replaceAll(">\\s*<", "><");
		
		page = page.replaceAll("\t", "");
		page = page.replaceAll("\b", "");
		page = page.replaceAll("\r", "");
		page = page.replaceAll("\n", "");
		page = page.replaceAll("\r\n", "");
		//BufferedWriter wr = new BufferedWriter( new FileWriter("C:\\newData2\\MediaIndonesia-FPI-"+(count++)+".txt"));
		//wr.write(page);
		//wr.close();
		
		Matcher newsItem = Pattern.compile("<div style=\"float:left; font-size:13px; line-height:17px; width:100%;\">(.*?)<div class=\"clr-line2\" style=\"margin-top:30px;\"></div>", Pattern.CASE_INSENSITIVE).matcher( page );
		
		Matcher newsItem2 = Pattern.compile("<div class=\"caption-read ml1\">(.*?)<div id=\"paginate-fotoread\" class=\"pagination-fotoread\">", Pattern.CASE_INSENSITIVE).matcher( page );
		

		
		
		while(newsItem.find())
		{
			System.out.println(count);
		//System.out.println(newsItem.group(1));
		String text = newsItem.group(1);
		
		text=text.replaceAll("<BR>","\r\n");
		text=text.replaceAll("<br>","\r\n");
		text=text.replaceAll("<br />", "\r\n");
		text = text.replaceAll("\\<.*?>","");
		text = text.replaceAll("[^\\p{ASCII}]", "");
		System.out.println(text);
		BufferedWriter wr2 = new BufferedWriter( new FileWriter("C:\\newData2\\Website4\\MediaIndonesia-"+key+"-"+(count++)+".txt"));
		wr2.write(text);
		wr2.close();
		
		//System.out.println(text);
	//	writer.write("http://newsbusters.org"+newsItem.group(1)+"\n");
		}
		

		while(newsItem2.find())
		{
			System.out.println(count);
		//System.out.println(newsItem.group(1));
		String text = newsItem2.group(1);
		text=text.replaceAll("<BR>","\r\n");
		text=text.replaceAll("<br>","\r\n");
		text=text.replaceAll("<br />", "\r\n");
		//System.out.println(text);
		text = text.replaceAll("\\<.*?>","");
		text = text.replaceAll("[^\\p{ASCII}]", "");
		System.out.println(text);
		BufferedWriter wr3 = new BufferedWriter( new FileWriter("C:\\newData2\\Website4\\MediaIndonesia-"+key+"-"+(count++)+".txt"));
		wr3.write(text);
		wr3.close();
		
		//System.out.println(text);
	//	writer.write("http://newsbusters.org"+newsItem.group(1)+"\n");
		}
		}
		catch(Exception e)
		{
			System.out.println(e);
		}
		
		}
		br.close();	
	       
		
}
	
	       }
	       
	    
	}
	
	private static String fetchPage(String urlString){
		StringBuilder buffer = new StringBuilder(INITIAL_BUF_SIZE);
		try {
			URL url = new URL(urlString);
			
			  HttpURLConnection.setFollowRedirects(false);
              HttpURLConnection urlConn = (HttpURLConnection)url.openConnection();

              urlConn.setRequestProperty("User-agent","Mozilla/2.0.0.11");
              urlConn.connect();
              BufferedReader reader = new BufferedReader(new InputStreamReader(urlConn.getInputStream()));

			//BufferedInputStream reader = new BufferedInputStream(url.openStream());
			int temp = reader.read();
			while(temp != -1){
				buffer.append((char)temp);
				temp = reader.read();
			}
		} catch (MalformedURLException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} catch (IOException e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		}
		return buffer.toString();
	}

}
